package nju.jun.reader.tieba.crawler;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import nju.jun.reader.tieba.NumberUtils;
import nju.jun.reader.tieba.config.TiebaConfigurationLoader;
import nju.jun.reader.tieba.model.TiebaNovel;
import nju.jun.reader.tieba.model.TiebaNovelChapter;

import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class TiebaHtmlParser {
	
	public static final String HOME_PAGE = "http://tieba.baidu.com";
	
	// ${0} should be the tieba name
	public static final String TIEBA_QUERY_TEMPLATE = "/f?ie=utf-8&kw=${0}";
	
	public String getTiebaQueryURL(TiebaNovel novel) {
		if (null == novel) {
			return null;
		}
		if (null == novel.getTiebaName()) {
			return null;
		}
		return HOME_PAGE + TIEBA_QUERY_TEMPLATE.replace("${0}", novel.getTiebaName());
	}
	
	private static final String THREAD_XPATH_ON_TOP = "//ul[@id='thread_list']/li[contains(@class,'thread_top')]/*/*/*/div[contains(@class, 'threadlist_text')]/a";
	private static final String THREAD_XPATH_NOT_ON_TOP = "//ul[@id='thread_list']/li/*/*/*/div[contains(@class, 'threadlist_text')]/a";
	
//	private static final String THREAD_TITLE = "title";
	private static final String THREAD_LINK = "href";
	
	private static final int MAX_THREADS_TO_PROCESS = 5;
	
	private static final String VOLUME_CHAPTER_PATTERN = "(第(.*?)卷(.*?))?第(.*?)章(.*)";
	
	/**
	 * 
	 * @param novel
	 * @return
	 * 	an array of TiebaNovelChapter
	 */
	public List<TiebaNovelChapter> getNovelChapters(TiebaNovel novel) {
		if (null == novel) {
			return null;
		}
		String tiebaUrl = getTiebaQueryURL(novel);
		try {
			Document doc = getDocument(tiebaUrl);
			
			String xpathStr;
			if (novel.isOnTop()) {
				xpathStr = THREAD_XPATH_ON_TOP;
			} else {
				xpathStr = THREAD_XPATH_NOT_ON_TOP;
			}
			XPath xpath = XPathFactory.newInstance().newXPath();
			NodeList targets =  (NodeList) xpath.evaluate(xpathStr, doc, XPathConstants.NODESET);
			if (targets == null) {
				return null;
			}
			
			List<TiebaNovelChapter> rtn = new ArrayList<TiebaNovelChapter>();
			// no more than 5 threads would be processed
			int num = Math.min(targets.getLength(), MAX_THREADS_TO_PROCESS);
			if (num == 0) {
				return null;
			}
			Pattern ptn = Pattern.compile(VOLUME_CHAPTER_PATTERN);
			for (int i = 0; i < num; i++) {
				Node target = targets.item(i);
				
				TiebaNovelChapter chapter = new TiebaNovelChapter();
				chapter.setNovel(novel);
				
//				target.getAttributes().getNamedItem(THREAD_TITLE).getNodeValue()
				String text = target.getTextContent();
				Matcher m = ptn.matcher(text);
				if (m.find()) {
					String volumeName = m.group(3).trim();
					if (null == volumeName) {
						volumeName = "";
					}
					chapter.setVolumeName(volumeName);
					int volumeNo = 0;
					try {
						volumeNo = NumberUtils.getNumberFromChineseString(m.group(2).trim());
					} catch (Exception e1) {
//						e1.printStackTrace();
					}
					chapter.setVolumeNumber(volumeNo);
					
					String title = m.group(4).trim();
					chapter.setChapterName(title);
					int chapterNo = -1;
					String chineseString = m.group(3).trim();
					try {
						chapterNo = NumberUtils.getNumberFromChineseString(chineseString);
					} catch (Exception e) {
						// TODO: use log here
//						System.err.println("Parse error or overflow for chapter " + chineseString + " whose title is " + title);
						System.err.println(e.getMessage() + " for text: " + text);
					}
					chapter.setChapterNumber(chapterNo);
				}
				String link = target.getAttributes().getNamedItem(THREAD_LINK).getNodeValue();
				chapter.setUrl(HOME_PAGE + link);
				
				if (!rtn.contains(chapter)) {
					rtn.add(chapter);
				}
			}
			return rtn;
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		} catch (XPathExpressionException e) {
			e.printStackTrace();
		}
		return null;
	}
	
	private static final String POST_CONTENT_XPATH = "//div[@id='j_p_postlist']/div[1]/*/*/*/div[@id[starts-with(., 'post_content')]]/text()";
	
	public void getNovelChapterContent(TiebaNovelChapter chapter) {
		if (null == chapter) {
			return;
		}
		try {
			Document doc = getDocument(chapter.getUrl());
			XPath xpath = XPathFactory.newInstance().newXPath();
			NodeList nodes =  (NodeList) xpath.evaluate(POST_CONTENT_XPATH, doc, XPathConstants.NODESET);
			String lineSep = System.getProperty("line.separator");
			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < nodes.getLength(); i++) {
				String line = nodes.item(i).getTextContent().trim();
				if (line.isEmpty()) {
					continue;
				}
				sb.append(line);
				sb.append(lineSep);
			}
			chapter.setContent(sb.toString());
			// TODO: what if the author post content in several posts?
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		} catch (XPathExpressionException e) {
			e.printStackTrace();
		}
	}
	
	@SuppressWarnings("deprecation")
	private Document getDocument(String url) throws MalformedURLException, IOException, ParserConfigurationException {
		HtmlCleaner cleaner = new HtmlCleaner();
		TagNode node = cleaner.clean(new URL(url), "GB18030");
		// TagNode node = cleaner.clean(new URL(tiebaUrl));
		CleanerProperties props = new CleanerProperties();
//		props.setAdvancedXmlEscape(false);
		// props.setRecognizeUnicodeChars(true);
		Document doc = new DomSerializer(props).createDOM(node);
		return doc;
	}

//	public String getNovelPageURL1(String tiebaUrl) {
//		if (null == tiebaUrl || tiebaUrl.isEmpty()) {
//			return null;
//		}
//
//		URL url;
//		try {
//			url = new URL(tiebaUrl);
//		} catch (MalformedURLException e) {
//			e.printStackTrace();
//			return null;
//		}
//
//		Reader reader = null;
//		StringBuffer sb = new StringBuffer();
//		try {
//			Test.testCharset(new URL(tiebaUrl).openStream());
//			reader = new InputStreamReader(url.openStream(), "GB18030");
//			// read html content
//			{
//				int read;
//				char[] cbuf = new char[1024];
//				while ((read = reader.read(cbuf)) != -1) {
//					sb.append(cbuf, 0, read);
//				}
//			}
//		} catch (IOException e) {
//			e.printStackTrace();
//			return null;
//		} finally {
//			if (null != reader) {
//				try {
//					reader.close();
//				} catch (IOException e) {
//					e.printStackTrace();
//				}
//			}
//		}
//		
//		return parseContent(sb.toString());
//	}
//	
//	private String parseContent(String content) {
//		// TODO: fill this method in
//		return null;
//	}

	public static void main(String[] args) {
		
		TiebaConfigurationLoader loader = new TiebaConfigurationLoader();
		loader.setConfigPath("config/tieba.xml");
		try {
			loader.parse();
		} catch (Exception e) {
			e.printStackTrace();
		}

//		TiebaHtmlParser parser = new TiebaHtmlParser();
//		Tieba tieba = Tieba.instance();
//		for (TiebaNovel novel : tieba.getNovels()) {
//			parser.getNovelPageURL1(parser.getTiebaQueryURL(novel));
//		}
		new Thread(new TiebaCrawler()).start();
	}
	
}
